"""
  Copyright (c) 2024 Northeastern University (China)
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
"""
# 主要任务：将每个OSS只留下自己独一无二的部分
# 输入：1.所有函数的索引
#      2.每个函数原始库的索引
# 输出：组件库，组件库中每个组件都是自己独一无二的函数
# A.划分策略：将oss中不是自己作为原始库的函数删掉
# B.划分策略:a.oss两两对比，S和OSS
#          b.发现重复函数，判断这个函数的原始库是不是S,如果是S，说明这个函数是S自己的就保留
#          c.如果发现函数属于特征库外部库的，直接删掉
#          d.如果函数原始库经前序判断是OSS，那就认为函数是S复制的OSS的，
#          e.最后如果S复制OSS的比例达到OSS的10%，就认为这部分函数极有可能不是S自己的，就从S中删除
#         注意：10%和目标代码检测是第一重阈值保持一致

import json
import sys
from datetime import datetime

import click

from src.common_utils import configure
import math
import multiprocessing
import re
import os
import time
import traceback
import io
from pathlib import Path
from tqdm import tqdm

from src.common_utils.utils_file import compress_dict
from src.operate_db.Component import Component as Component
from src.operate_db.InitialSig import InitialSig as InitialSig
from src.common_utils.logger import logger

# 当前文件的路径
g_cur_dir = os.path.dirname(os.path.abspath(__file__))
# 当前文件的父路径
father_path = os.path.dirname(g_cur_dir)
sys.path.append(father_path)
theta=0.1
class CodeSegmentation:
    oss_sigdb=InitialSig()
    component=Component()
    alloss={}
    aveFuncs={}
    # func_info_all_noderes_path = Path(g_cur_dir, "data_out", "func_info_all.json")  # 记录中间数据
    # func_origin_all_res_path = Path(g_cur_dir, "data_out", "func_origin_all_res.json")  # 记录的每个函数原始库信息的数据
    func_info_all_noderes_path = Path(configure.CONFIG['component_construct'], "func_info_all.json")  # 记录中间数据
    func_origin_all_res_path = Path(configure.CONFIG['component_construct'], "func_origin_all_res.json")  # 记录的每个函数原始库信息的数据

    def __init__(self):
        print(f"加载信息")
        func_info_all={}
        func_origin_data={}
        res=self.oss_sigdb.get_data({},{'sig_full_name':1,'sig_info_cnt':1,'ver_cnt':1})
        self. aveFuncs={sig['sig_full_name']:int(sig['sig_info_cnt']/sig['ver_cnt']) for sig in res}
        manager = multiprocessing.Manager()
        # self.uniqueFuncs = manager.dict({})
        # with open(self.func_info_all_noderes_path, 'r', encoding='utf-8') as file:
        #     # 逐行读取 JSON 数据
        #     for line in file:
        #         self.uniqueFuncs.update(json.loads(line))
        # print(f'{len(self.uniqueFuncs)}')
        with open(self.func_origin_all_res_path, 'r', encoding='utf-8') as file:
            # 逐行读取 JSON 数据
            for line in file:
                func_origin_data.update(json.loads(line))
        self.func_origin = manager.dict(func_origin_data)
    def get_files_in_first_level(self, directory):
            """
            这是寻找一个组件源码文件中的顶层配置文件，readme,license copying
            @param directory: 组件源码文件路径
            @return: files_in_first_level list
            """
            try:
                # 获取指定目录下的所有文件和目录
                entries = os.listdir(directory)

                # 定义你感兴趣的文件名模式（不区分大小写）
                desired_file_pattern = re.compile(r'(readme|license|copying)', re.I)

                # 筛选出特定文件名的文件（不区分大小写），包含上述三个文件名的任何一个或它们的变体
                files_in_first_level = [entry for entry in entries if
                                        os.path.isfile(os.path.join(directory, entry)) and desired_file_pattern.search(
                                            entry)]

                return files_in_first_level
            except Exception as e:
                print("获取第一层目录中的文件相对路径时出现错误:", str(e))
                return None
    def compare_date_time(self,S,OSS):
        OSS_time = datetime.strptime(OSS, "%Y-%m-%d %H:%M:%S")
        S_time=datetime.strptime(OSS, "%Y-%m-%d %H:%M:%S")
        if OSS_time.year < 2019 and S_time.year<2019:
            if OSS_time <S_time:
                return True
            else:
                return False
        elif OSS_time.year<2019 and S_time.year>=2019:
            return False
        elif OSS_time.year>=2019 and S_time.year<2019:
            return True
        elif OSS_time.year>=2019 and S_time.year>=2019:
            if OSS_time <S_time:
                return True
            else:
                return False

    def find_TPL_func(self, metadata, S):
        """
        寻找oss中属于其他TPL的部分
        @param metadata: 表示 S的所有函数的签名
        @param S:表示OSS的名称
        @return:candix{名称：条数}temp{名称：[函数]}
        """
        candiX={}
        temp={}
        removefuncs = []
        for func_hash in metadata:
            if len(self.uniqueFuncs[func_hash]) != 1:
                for OSS in self.uniqueFuncs[func_hash]:
                    if OSS == S:
                        continue
                    if OSS not in candiX:  # 表示这个oss还没被记录成s的候选可能成员X
                        temp[OSS] = []
                        candiX[OSS] = 0
                    try:
                        if func_hash in self.func_origin:
                            appears = [origin[0] for origin in self.func_origin[func_hash]]
                            if "external lib" in appears:
                                removefuncs.append(func_hash)
                                continue
                            elif S in appears:
                                continue
                            elif OSS in appears:  # 说明oss是源不用在做其他对比了
                                candiX[OSS] += 1
                                temp[OSS].append(func_hash)
                            elif self.compare_date_time(self.uniqueFuncs[func_hash][S][
                                0],self.uniqueFuncs[func_hash][OSS][0]):  # 注意：这里还是用func_hash，s_hash只起比较分数作用
                                candiX[OSS] += 1
                                temp[OSS].append(func_hash)
                        elif self.compare_date_time(self.uniqueFuncs[func_hash][S][0],self.uniqueFuncs[func_hash][OSS][0]):  # 注意：这里还是用func_hash，s_hash只起比较分数作用
                            candiX[OSS] += 1
                            temp[OSS].append(func_hash)
                    except Exception as e:
                        print(f"错误：{S}错误")
                        traceback.print_exc()
        return  candiX, temp,removefuncs
    # def find_remove_func(self,metadata,S):
    #     candix,temp,removefuncs=self.find_TPL_func(metadata,S)
    #     for x in candix:
    #         if self.aveFuncs[x] == 0:
    #             continue
    #         if self.aveFuncs[x]*theta<=self.aveFuncs[S]:#说明S是可以达到x的10%的
    #             if float(candix[x]/float(self.aveFuncs[x]))>=theta:
    #                 removefuncs.extend(temp[x])
    #                 print(f"S比X大，{S},候选：{x}:{candix[x]}")
    #         else :#说明s太小，总量都没有达到x的10%
    #             if self.aveFuncs[S]==0:
    #                 continue
    #             elif float(candix[x]/float(self.aveFuncs[S]))>=theta:
    #                 removefuncs.extend(temp[x])
    #                 print(f"X比S大，{S},候选：{x}:{candix[x]}")
    #     return removefuncs


    def  find_remove_func(self,metadata,S):
        """
        寻找oss需要删掉的函数
        @param metadata:oss所有函数的哈希和对应的信息 dict
        @param S: oss的名称 str
        @return:removefuncs 需要删掉的函数哈希 list
        """
        removefuncs=[]
        for func_hash in metadata:
            try:
                if func_hash in self.func_origin:
                    appears=[origin[0] for origin in self.func_origin[func_hash]]
                    if S in appears:
                        pass
                    else:
                        #说明这个函数是s克隆的别人的
                        removefuncs.append(func_hash)

            except Exception as e:
                print(f"错误：{S}错误")
                traceback.print_exc()
        return removefuncs

    def code_segmentation(self,S,metadata):
        """
        划分组件代表性特征的核心函数
        生成组件库的核心函数
        @param S: 开源软件oss名称 str
        @param metadata: 初始签名库initialsig库中存储的该oss详细信息 dict
        @return: 修改数据库
        """
        logger.info("[+]code_segmentation")
        store_data = metadata  # 注意，这里使用copy，防止更改原来的数据
        removedFuncs=self.find_remove_func(metadata['sig_info'],S)
        if len(removedFuncs)==0:
            #S不需要代码分割，已经是一个原子的OSS
           pass
        else:
            removedFuncs = set(removedFuncs)
            for func_hash in removedFuncs:
                del store_data['sig_info'][func_hash]
        # 不要在之前的数据库上改，要将数据添加到新的数据库中
        store_data['func_feature']=store_data['sig_info']
        store_data['func_feature_cnt']=len(store_data['func_feature'])
        store_data['ave_func'] = self.aveFuncs[S]
        if 'sig_info' in store_data:
            del store_data['sig_info']
        del store_data['_id']
        func_dict=store_data['func_feature']
        if len(func_dict) != 0:
            sig_info_zip = compress_dict(func_dict)
            store_data['func_feature']=sig_info_zip
            threshold = 15800000  # 默认为15.8MB
            bson_size = len(sig_info_zip)
            if bson_size <= threshold:
                pass
            else:
                # 使用GridFS存储
                sig_info_id = self.component.use_Gridfs_store(io.BytesIO(sig_info_zip),store_data['sig_full_name'] + ".gz")
                store_data['func_feature']=sig_info_id
            if  store_data['ave_func']!=0:#平均条数>0再存储，存到组件库的必须是完全有效的
                res=self.component.get({'sig_full_name':S},{"_id":1})
                if len(res)!=0:
                    self.component.update_one_with_id(res[0]['_id'],store_data)
                else:
                    self.component.add_meta(store_data)

    def split_list(self, list, n):
        """
        将数据n等分，分发给n个线程执行

        :param list: 原始完整list
        :param n: 切分数量，即线程数量
        :return: 切分后的list起始位置
        """
        step_count = math.ceil(len(list) / n)
        for i in range(0, len(list), step_count):
            end = i + step_count
            if end > len(list):
                end = len(list)
            # print(f"yield: {i}:{end}")
            yield list[i:end]

    def process_oss(self, idx,list_arr):
        """
        oss划分代表性特征的主函数
        进程执行的子函数
        @param idx: 进程编号 int
        @param list_arr: 进程需要处理的任务列表 list
        @return: 修改数据库
        """
        for sig in tqdm(list_arr,total=len(list_arr),desc=f"thread{idx}处理oss进度"):
            S=sig['sig_full_name']
            ossinfo = self.oss_sigdb.get_one_sig({'sig_full_name': S},{"sig_full_name": 1, "sig_info": 1, "oss_url": 1,"sig_info_cnt": 1, "ver_cnt": 1,"src_local_path": 1,'repo_name':1,'oss_name':1,'oss_source':1})[0]
            try:
                self.code_segmentation( S, ossinfo)
            except Exception as e:
                logger.error('[+]  An error occurred', exc_info=True)
                traceback.print_exc()
                continue



    def create_component_db(self,num_processes,all_component_name=None):
        """
        构建组件库的主函数
        多进程执行
        @param num_processes: 并发进程数 int
        @param all_component_name: 生成组件库的oss名称 str list
        @return: 修改数据库
        """
        logger.info("[+] code segmentation")
        processes = []
        if all_component_name==None:
            all_component_name = self.get_all_signame()
        for idx, list_arr in enumerate(self.split_list(list(all_component_name), num_processes)):  # 分割任务列表并获取进程编号
            process = multiprocessing.Process(target=self.process_oss, args=(idx + 1, list_arr))
            processes.append(process)
        for process in processes:
            process.start()
        for process in processes:
            process.join()  # 等待所有子进程结束后再执行主进程的代码
    def addtional_oss_metafile(self):
        """
        找每个组件顶层的配置文件，为之后识别结果验证做准备
        @return: 修改数据库，向组件库添加metafile字段
        """
        oss = self.oss_sigdb.get_data({}, {"src_local_path": 1, 'sig_full_name': 1})
        for data in tqdm(oss, total=len(oss), desc="处理进度"):
            oss_src_path = os.path.join(configure.CONFIG['local_data_path'], data['src_local_path'])  # 组件源码仓库
            metadata_file = self.get_files_in_first_level(oss_src_path)
            if metadata_file is not None:
                data_store = {
                    'src_local_path': data['src_local_path'],
                    'metafile': metadata_file
                }
                self.component.update_one_with_condition({'sig_full_name': data['sig_full_name']}, data_store)

    def get_all_signame(self):
        all_component_name = self.oss_sigdb.collection.find({}, {"sig_full_name": 1,"_id":0,"sig_info_cnt":1}).sort('sig_info_cnt', 1)
        return all_component_name
@click.command()
@click.option('--cpu', type=int, default=8, help='并发执行数')
def main(cpu):
    start = time.time()
    codeSegmentation = CodeSegmentation()
    # codeSegmentation.addtional_oss_metafile()
    codeSegmentation.create_component_db(num_processes=cpu)
    end = time.time()
    print(f"共计用时：{end - start}")
if __name__ == "__main__":
   main()

